<?php
	define('ENTRIES_PER_PAGE', 20);
	define('CATEGORY', 'cat:cs.*');
?>
<?php

	set_time_limit(0);

	include_once('libs/xmlparser.lib.php');
	include_once('libs/mysql.lib.php');
	include_once('libs/hints.php');	
	include_once('libs/insert_builder.php');
	
	$dbconf = array (
		'host' 		=> "localhost",
		'port' 		=> "3306",
		'db_name' 	=> "arxiv_crawler",
		'user' 		=> "arxiv_crawler",
		'password' 	=> "arxiv_crawler"
	);
	$db = new Mysql($dbconf);
	$db->ExecuteQuery('TRUNCATE TABLE entries');
	$db->ExecuteQuery('TRUNCATE TABLE links');
	$db->ExecuteQuery('TRUNCATE TABLE categories');
	$db->ExecuteQuery('TRUNCATE TABLE authors');
	
	$insert_builder_obj = new insert_builder();

	for ($page = 0; $page < $_REQUEST['pages'] || $_REQUEST['pages'] == 0 ; $page++) {
		
		$start = $_REQUEST['page'] * ENTRIES_PER_PAGE;
		$url = "http://export.arxiv.org/api/query?search_query=".CATEGORY."&start=$start&max_results=".ENTRIES_PER_PAGE;

		$xmlparser = new XMLParser($url);
		$data = $xmlparser->result;
		
		if (!e($data)) {
			break;
		}
		
		foreach ($data['feed']['entry'] as $item) {
		
			$entry = array(
				'arxiv_id' => $item['id'],
				'updated' => $item['updated'],
				'published' => $item['published'],
				'title' => $item['title'],
				'summary' => $item['summary']
			);
			
			$db->Insert("entries", array_keys($entry), array_values($entry));
			$arr = $db->Select("entries", "MAX(id) AS last_id");

			$authors = array();
			foreach($item['author'] as $author) {
				if (e($author['name'])) {
					$authors[] = array(
						'name' => $author['name'],
						'entry_id' => $arr[0]['last_id']
					);
				}
			}
			
			
			//сохраняем_авторов
			if (e($authors)) {
				$insert_builder_obj->set_table('authors');
				$insert_builder_obj->set_fields(array_keys($authors[0]));
				foreach ($authors as $author) {
					$insert_builder_obj->add_values(array_values($author));
				}
				$db->ExecuteQuery($insert_builder_obj->returnSQL());
				$insert_builder_obj->reset();
			}

			$links = array();
			foreach($item['link'] as $link) {
				if (!empty($link)) {
					$links[] = array(
						'title' => (e($link['title'])) ? $link['title'] : '',
						'href' => (e($link['href'])) ? $link['href'] : '',
						'rel' => (e($link['rel'])) ? $link['rel'] : '',
						'type' => (e($link['type'])) ? $link['type'] : '',
						'entry_id' => $arr[0]['last_id']
					);
				}
			}
			
			//сохраняем_ссылки
			if (e($links)) {
				$insert_builder_obj->set_table('links');
				$insert_builder_obj->set_fields(array_keys($links[0]));
				foreach ($links as $link) {
					$insert_builder_obj->add_values($link);
				}
				$db->ExecuteQuery($insert_builder_obj->returnSQL());
				$insert_builder_obj->reset();
			}

			$categories = array();
			foreach($item['category'] as $category) {
				if (e($category['term'])) {
					$categories[] = array(
						'term' => $category['term'],
						'entry_id' => $arr[0]['last_id']
					);
				}
			}
			
			//сохраняем_категории
			if (e($categories)) {
				$insert_builder_obj->set_table('categories');
				$insert_builder_obj->set_fields(array_keys($categories[0]));
				foreach ($categories as $category) {
					$insert_builder_obj->add_values(array_values($category));
				}
				$db->ExecuteQuery($insert_builder_obj->returnSQL());
				$insert_builder_obj->reset();
			}
			
			
			_log ( 'Entry #' . $entry['arxiv_id']. '(' . $arr[0]['last_id'] . ') has been saved;' );
		}

	}
?>